from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
import time
import redis

redis_conn = redis.Redis(host='127.0.0.1', port=6379)
driver = webdriver.PhantomJS()
# driver  = webdriver.Firefox(executable_path='C:\Program Files (x86)\Mozilla Firefox\geckodriver.exe')
driver.get('https://m.dianping.com/shoplist/852/d/1/c/10/s/s_-1?from=m_nav_1_meishi')
time.sleep(3)  #加载js内容需要时间
count = 0
temp = True
while temp:   #下滑到底部后还能继续加载数据，然后继续下滑。下滑后会覆盖前一页js生成的数据
    soup = BeautifulSoup(driver.page_source,'lxml')
    shop_list = soup.find(class_="list-search")
    page_url = shop_list.find_all('li',class_='list-item border-bottom-new')
    for url in page_url:
        url = url.find('a')
        url = url.get('href')
        url = 'http://m.dianping.com%s'%url
        print(url)
        res = redis_conn.sadd('test_set3',url)
        if res ==0:
            count+=1
            time.sleep(0.05)
        else:
            count = 0

    driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")  # 下滑到底部
    time.sleep(3)
    if count >2000:
        temp = False
        print(count)





